{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3PzJTYKBxZHE",
    "outputId": "0a79f666-561e-4d26-d25b-ce7de19ba303"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'llmware'...\n",
      "remote: Enumerating objects: 5664, done.\u001b[K\n",
      "remote: Counting objects: 100% (1548/1548), done.\u001b[K\n",
      "remote: Compressing objects: 100% (679/679), done.\u001b[K\n",
      "remote: Total 5664 (delta 903), reused 1115 (delta 838), pack-reused 4116\u001b[K\n",
      "Receiving objects: 100% (5664/5664), 532.23 MiB | 23.42 MiB/s, done.\n",
      "Resolving deltas: 100% (3142/3142), done.\n",
      "Updating files: 100% (468/468), done.\n",
      "/content/llmware\n",
      "Processing /content/llmware\n",
      "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "Collecting e\n",
      "  Downloading e-1.4.5.tar.gz (1.8 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "Collecting boto3==1.24.53 (from llmware==0.2.14)\n",
      "  Downloading boto3-1.24.53-py3-none-any.whl (132 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.5/132.5 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.20.3)\n",
      "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (1.25.2)\n",
      "Collecting openai>=1.0.0 (from llmware==0.2.14)\n",
      "  Downloading openai-1.30.1-py3-none-any.whl (320 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pymongo>=4.7.0 (from llmware==0.2.14)\n",
      "  Downloading pymongo-4.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (670 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.0/670.0 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.19.1)\n",
      "Requirement already satisfied: torch>=1.13.1 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (2.2.1+cu121)\n",
      "Requirement already satisfied: transformers>=4.36.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (4.40.2)\n",
      "Collecting Wikipedia-API==0.6.0 (from llmware==0.2.14)\n",
      "  Downloading Wikipedia_API-0.6.0-py3-none-any.whl (14 kB)\n",
      "Collecting psycopg-binary==3.1.17 (from llmware==0.2.14)\n",
      "  Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting psycopg==3.1.17 (from llmware==0.2.14)\n",
      "  Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pgvector==0.2.4 (from llmware==0.2.14)\n",
      "  Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n",
      "Collecting colorama==0.4.6 (from llmware==0.2.14)\n",
      "  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
      "Collecting einops==0.7.0 (from llmware==0.2.14)\n",
      "  Downloading einops-0.7.0-py3-none-any.whl (44 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.10.2.post1)\n",
      "Collecting botocore<1.28.0,>=1.27.53 (from boto3==1.24.53->llmware==0.2.14)\n",
      "  Downloading botocore-1.27.96-py3-none-any.whl (9.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.3/9.3 MB\u001b[0m \u001b[31m29.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3==1.24.53->llmware==0.2.14)\n",
      "  Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
      "Collecting s3transfer<0.7.0,>=0.6.0 (from boto3==1.24.53->llmware==0.2.14)\n",
      "  Downloading s3transfer-0.6.2-py3-none-any.whl (79 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.8/79.8 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware==0.2.14) (4.11.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from Wikipedia-API==0.6.0->llmware==0.2.14) (2.31.0)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (3.14.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (2023.6.0)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (4.66.4)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (6.0.1)\n",
      "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (24.0)\n",
      "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (3.0.1)\n",
      "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.11.4)\n",
      "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.2.2)\n",
      "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.4.2)\n",
      "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (4.4.2)\n",
      "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.58.1)\n",
      "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.12.1)\n",
      "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.8.1)\n",
      "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.3.7)\n",
      "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.4)\n",
      "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.0.8)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.0.0->llmware==0.2.14) (1.7.0)\n",
      "Collecting httpx<1,>=0.23.0 (from openai>=1.0.0->llmware==0.2.14)\n",
      "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (2.7.1)\n",
      "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (1.3.1)\n",
      "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware==0.2.14)\n",
      "  Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m34.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (1.12)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (3.3)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (3.1.4)\n",
      "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
      "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
      "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
      "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
      "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
      "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
      "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
      "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
      "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
      "Collecting nvidia-nccl-cu12==2.19.3 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
      "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
      "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (2.2.0)\n",
      "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.1->llmware==0.2.14)\n",
      "  Using cached nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware==0.2.14) (2023.12.25)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware==0.2.14) (0.4.3)\n",
      "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware==0.2.14) (3.7)\n",
      "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware==0.2.14) (1.2.1)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14) (2.8.2)\n",
      "Collecting urllib3<1.27,>=1.25.4 (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14)\n",
      "  Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14) (2024.2.2)\n",
      "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14)\n",
      "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14)\n",
      "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware==0.2.14) (0.41.1)\n",
      "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware==0.2.14) (4.2.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware==0.2.14) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware==0.2.14) (2.18.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->Wikipedia-API==0.6.0->llmware==0.2.14) (3.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware==0.2.14) (3.5.0)\n",
      "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware==0.2.14) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.1->llmware==0.2.14) (2.1.5)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.1->llmware==0.2.14) (1.3.0)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware==0.2.14) (2.22)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14) (1.16.0)\n",
      "Building wheels for collected packages: e, llmware\n",
      "  Building wheel for e (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for e: filename=e-1.4.5-py3-none-any.whl size=2793 sha256=ff3f471fbc464db8eff967277fa643ec0ba3b94c10c1644a6e15f8b01e908626\n",
      "  Stored in directory: /root/.cache/pip/wheels/3d/dc/3e/208e61209d7750644677d72934d2997177aa34e894bd398e16\n",
      "  Building wheel for llmware (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for llmware: filename=llmware-0.2.14-py3-none-any.whl size=56031832 sha256=ea11b7471e405abe341532f51f666e25979276045dfeed4123f7667ee2b1c13e\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-2tk0k1g5/wheels/ff/a8/4a/2288aab3d94134f03a5bcbe7f6b55020b68ca718e0dc78323b\n",
      "Successfully built e llmware\n",
      "Installing collected packages: e, urllib3, psycopg-binary, psycopg, pgvector, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, jmespath, h11, einops, dnspython, colorama, pymongo, nvidia-cusparse-cu12, nvidia-cudnn-cu12, httpcore, botocore, Wikipedia-API, s3transfer, nvidia-cusolver-cu12, httpx, openai, boto3, llmware\n",
      "  Attempting uninstall: urllib3\n",
      "    Found existing installation: urllib3 2.0.7\n",
      "    Uninstalling urllib3-2.0.7:\n",
      "      Successfully uninstalled urllib3-2.0.7\n",
      "Successfully installed Wikipedia-API-0.6.0 boto3-1.24.53 botocore-1.27.96 colorama-0.4.6 dnspython-2.6.1 e-1.4.5 einops-0.7.0 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 jmespath-1.0.1 llmware-0.2.14 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 openai-1.30.1 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.2 s3transfer-0.6.2 urllib3-1.26.18\n",
      "Collecting faiss-cpu\n",
      "  Downloading faiss_cpu-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (27.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.0/27.0 MB\u001b[0m \u001b[31m43.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from faiss-cpu) (1.25.2)\n",
      "Installing collected packages: faiss-cpu\n",
      "Successfully installed faiss-cpu-1.8.0\n"
     ]
    }
   ],
   "source": [
    "!pip install llmware\n",
    "!pip install faiss-cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mwADpGQ9x1Kd"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from llmware.setup import Setup\n",
    "from llmware.library import Library\n",
    "from llmware.prompts import Prompt, HumanInTheLoop\n",
    "from llmware.retrieval import Query\n",
    "from llmware.configs import LLMWareConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qisovkCXygD9"
   },
   "outputs": [],
   "source": [
    "LLMWareConfig().set_active_db(\"sqlite\")\n",
    "\n",
    "llm = \"llmware/dragon-yi-6b-gguf\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ksZy1eJazIeu"
   },
   "outputs": [],
   "source": [
    "local_path = Setup().load_sample_files()\n",
    "agreements_path = os.path.join(local_path, \"AgreementsLarge\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "20K71ig1zU2T",
    "outputId": "23a519f3-7171-4e96-e34f-31ae19857620"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Starting:  Parsing 'AgreementsLarge' Folder\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'docs_added': 81,\n",
       " 'blocks_added': 12552,\n",
       " 'images_added': 0,\n",
       " 'pages_added': 1345,\n",
       " 'tables_added': 10,\n",
       " 'rejected_files': []}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(f\"\\nStarting:  Parsing 'AgreementsLarge' Folder\")\n",
    "msa_lib = Library().create_new_library(\"example6_library\")\n",
    "msa_lib.add_files(agreements_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "djgIpiUI0jpF",
    "outputId": "6f538534-c4ec-430e-8ad6-bca34e87e3eb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Completed Parsing - now, let's look for the 'master service agreements', e.g., 'msa'\n"
     ]
    }
   ],
   "source": [
    "print(f\"\\nCompleted Parsing - now, let's look for the 'master service agreements', e.g., 'msa'\")\n",
    "\n",
    "q = Query(msa_lib)\n",
    "query = '\"master services agreement\"'\n",
    "results = q.text_search_by_page(query, page_num=1, results_only=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cRUgy6YH029j"
   },
   "outputs": [],
   "source": [
    "msa_docs = results[\"file_source\"]\n",
    "msa_doc_ids = results[\"doc_ID\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 176,
     "referenced_widgets": [
      "750db138aba04233b7552a809fe028e7",
      "73f01dd1f62148ccbc03fd953927076b",
      "0c6b1a2f4b6a4602bf5760d9e515f422",
      "3bb7c2beaf114d14a0b6e63947c25688",
      "58e60eec487640b3a552761db277c105",
      "1abee224a04142a798cc9f976fde5f41",
      "e96669f232d04c079c91a1121e7dedb5",
      "0e66bf7f285343ea98f0456929049e72",
      "3a5c51a6dd8e4f8a9fda6f11d370529a",
      "93a0447906784d72b7b2b4a5971c66d9",
      "a94fb32c43794d6b87f15c22714c68f1"
     ]
    },
    "id": "doJyVwSQ09iJ",
    "outputId": "aa7bf1d0-a0b6-4e74-a000-6a5cb2a3ef79"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
      "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
      "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
      "You will be able to reuse this secret in all of your notebooks.\n",
      "Please note that authentication is recommended but still optional to access public models or datasets.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "750db138aba04233b7552a809fe028e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "dragon-yi-6b-q4_k_m.gguf:   0%|          | 0.00/3.67G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prompter = Prompt().load_model(llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "sguPV2q-1Fef",
    "outputId": "fdf3fef2-9dd3-4fa8-b2de-d37acd700e6e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "update: identified the following msa doc id:  [2, 3, 6, 10, 11, 14, 24, 27, 30, 49, 50, 51, 52, 53, 55, 56, 61, 62, 65, 66]\n",
      "\n",
      "\n",
      "1 Reviewing MSA -  2 Testco MSA Zeta.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices will be \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be   illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)   year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1100, 'output': 6, 'total': 1106, 'metric': 'tokens', 'processing_time': 585.4072885513306}, 'time_stamp': 'Tue May 21 16:22:25 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2414, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3217, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3217, 'evidence_stop_char': 3619, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Eta.pdf': [5, 4, 6], 'Testco MSA Zeta.pdf': [5, 4]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "2 Reviewing MSA -  3 Testco MSA Theta.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be   illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 416.88949728012085}, 'time_stamp': 'Tue May 21 16:29:22 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1607, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1607, 'evidence_stop_char': 2010, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2010, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Theta.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "3 Reviewing MSA -  6 Testco MSA Eta.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices will be \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be   illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 428.66738963127136}, 'time_stamp': 'Tue May 21 16:36:31 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2414, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Eta.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "4 Reviewing MSA -  10 Testco MSA Zeta.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)   year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 420.84798073768616}, 'time_stamp': 'Tue May 21 16:43:32 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1207, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2815, 'source_name': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Zeta.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "5 Reviewing MSA -  11 Testco MSA Theta.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be   illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 420.8565924167633}, 'time_stamp': 'Tue May 21 16:50:33 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1607, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1607, 'evidence_stop_char': 2010, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2010, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Theta.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Theta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "6 Reviewing MSA -  14 Testco MSA Eta.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices will be \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be   illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 431.8014826774597}, 'time_stamp': 'Tue May 21 16:57:45 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2414, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Eta.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "7 Reviewing MSA -  24 Testco MSA Lambda.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices will be \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be   illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 428.2654275894165}, 'time_stamp': 'Tue May 21 17:04:53 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Lambda.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Lambda.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Lambda.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Lambda.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Lambda.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2414, 'source_name': 'Testco MSA Lambda.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'Testco MSA Lambda.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Lambda.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Lambda.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Lambda.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "8 Reviewing MSA -  27 Testco MSA Omega.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  yea r from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n  Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices will be \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 769, 'output': 6, 'total': 775, 'metric': 'tokens', 'processing_time': 420.37582778930664}, 'time_stamp': 'Tue May 21 17:11:53 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1207, 'source_name': 'Testco MSA Omega.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Omega.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2815, 'source_name': 'Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Omega.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "9 Reviewing MSA -  30 Testco MSA Omicron.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n  Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices will be \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be   illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 426.8950161933899}, 'time_stamp': 'Tue May 21 17:19:00 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Omicron.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1607, 'source_name': 'Testco MSA Omicron.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1607, 'evidence_stop_char': 2010, 'source_name': 'Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2010, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Omicron.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "10 Reviewing MSA -  49 Testco MSA Sigma.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n or  any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)   year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\n Agreement must be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 768, 'output': 6, 'total': 774, 'metric': 'tokens', 'processing_time': 424.2683684825897}, 'time_stamp': 'Tue May 21 17:26:05 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Sigma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 808, 'source_name': 'Testco MSA Sigma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 808, 'evidence_stop_char': 1211, 'source_name': 'Testco MSA Sigma.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1211, 'evidence_stop_char': 1612, 'source_name': 'Testco MSA Sigma.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1612, 'evidence_stop_char': 2022, 'source_name': 'Testco MSA Sigma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2022, 'evidence_stop_char': 2424, 'source_name': 'Testco MSA Sigma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2424, 'evidence_stop_char': 2826, 'source_name': 'Testco MSA Sigma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Sigma.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement ', 'match_score': 1.0, 'source': 'Testco MSA Sigma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement ', 'match_score': 1.0, 'source': 'Testco MSA Sigma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "11 Reviewing MSA -  50 Testco MSA Xi.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)   year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 419.32227754592896}, 'time_stamp': 'Tue May 21 17:33:04 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Xi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Xi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1207, 'source_name': 'Testco MSA Xi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Xi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Xi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Xi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Xi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Xi.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Xi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Xi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "12 Reviewing MSA -  51 Testco MSA Delta.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nof the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and  numbers indicated in this section. Notices will be deemed delivered \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 419.91788125038147}, 'time_stamp': 'Tue May 21 17:40:05 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Delta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1609, 'source_name': 'Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1609, 'evidence_stop_char': 2019, 'source_name': 'Testco MSA Delta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2019, 'evidence_stop_char': 2421, 'source_name': 'Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2421, 'evidence_stop_char': 2823, 'source_name': 'Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Delta.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "13 Reviewing MSA -  52 Testco MSA Pi.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  yea r from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 769, 'output': 6, 'total': 775, 'metric': 'tokens', 'processing_time': 424.0798804759979}, 'time_stamp': 'Tue May 21 17:47:09 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1207, 'source_name': 'Testco MSA Pi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Pi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2815, 'source_name': 'Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Pi.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "14 Reviewing MSA -  53 Testco MSA Psi.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be   illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 414.80505204200745}, 'time_stamp': 'Tue May 21 17:54:04 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Psi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1607, 'source_name': 'Testco MSA Psi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1607, 'evidence_stop_char': 2010, 'source_name': 'Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2010, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Psi.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "15 Reviewing MSA -  55 Testco MSA Mu.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)   year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 416.79119968414307}, 'time_stamp': 'Tue May 21 18:01:01 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1207, 'source_name': 'Testco MSA Mu.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Mu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Mu.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "16 Reviewing MSA -  56 Testco MSA Nu.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)   year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 420.5427143573761}, 'time_stamp': 'Tue May 21 18:08:02 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1207, 'source_name': 'Testco MSA Nu.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Nu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2815, 'source_name': 'Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Nu.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "17 Reviewing MSA -  61 Testco MSA Beta.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)   year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\nof the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices will be deemed \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 765, 'output': 6, 'total': 771, 'metric': 'tokens', 'processing_time': 412.129132270813}, 'time_stamp': 'Tue May 21 18:14:54 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Beta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1607, 'source_name': 'Testco MSA Beta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1607, 'evidence_stop_char': 2010, 'source_name': 'Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2010, 'evidence_stop_char': 2412, 'source_name': 'Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Beta.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "18 Reviewing MSA -  62 Testco MSA Gamma.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\n Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices will be deemed \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 413.32629108428955}, 'time_stamp': 'Tue May 21 18:21:47 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Gamma.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1609, 'source_name': 'Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1609, 'evidence_stop_char': 2016, 'source_name': 'Testco MSA Gamma.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2016, 'evidence_stop_char': 2418, 'source_name': 'Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2418, 'evidence_stop_char': 2820, 'source_name': 'Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Gamma.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "19 Reviewing MSA -  65 Testco MSA Phi.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)   year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,  the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 421.4450001716614}, 'time_stamp': 'Tue May 21 18:28:49 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1207, 'source_name': 'Testco MSA Phi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'Testco MSA Phi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2413, 'source_name': 'Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2413, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Phi.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n",
      "\n",
      "\n",
      "20 Reviewing MSA -  66 Testco MSA Tau.pdf\n",
      "update: llm response -  {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \" party  may  be   deemed  notice  of   termination  of  this  Agreement,  effective  on  the  date  of  assignment, by the other party.   Survival upon Termination or End of Term.   The   provisions  regarding  warranty,  limitation  of  liability,  confidentiality,  fees  and   expenses,  obligations  on   termination or expiration, ownership and license, and miscellaneous of this  Agreement, and \\n any  work  order  or  services  description  if  the  other  party  is  in  material  breach  or   default  of any obligation that is not cured  within 15 days' notice of such breach. The TestCo agrees to pay all fees   for services performed  and  expenses  incurred  prior  to  the  termination  of  this  Agreement.  Termination  of  this   Agreement  will  terminate  all  outstanding  Statement \\n of  the  Customer, which  includes  trade  secrets.   Accordingly,  in  order  to  prevent TestCo  and  its  employees  from  intentionally  or  unintentionally   misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1)  year from the termination of this Agreement, its employees who provide services hereunder will not   work on any Competitive Products \\n below.   Each of the contracting parties agrees to notify the other in writing of address or  contact changes.     All  notices,  authorizations,  and  requests  given  or  made  in  connection  with  this  agreement,  including notice of   termination of this agreement, must be sent by  post, express courier, facsimile,  or email to the addresses and   numbers indicated in this section. Notices \\nmust be   brought within two years from the date that the cause of action arose.   Term and Termination of Agreement; Assignment.   This Agreement shall remain in effect until terminated. Either party may terminate this agreement,  any Statement of   Work or Services Description for convenience by giving the other party 30 days   written  notice.  Either  party  may   terminate  this  Agreement  or \\nany provisions specified  as surviving in a Statement of Work or Services  Description, survive any termination or expiration of this agreement,   any Statement of Work or   Services Description.   Severability.  If  a  court  holds  any   provision  of  this  Agreement  to  be  illegal,  invalid  or  unenforceable,  the   remaining   provisions will remain in full force and effect and the parties \\n(24) months  following   termination  of  this  Agreement,  directly  or  indirectly,  call  on  or  attempt  to  call  on,  hire,  solicit,  or  induce  any   change  in  or  cessation  of,   the  business  relationship,  of  any   customers,  clients, contractors, vendors, contract   manufacturers, suppliers, investors or employees of other on  whom  the party  called  on  or  became  acquainted \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 767, 'output': 6, 'total': 773, 'metric': 'tokens', 'processing_time': 437.4010033607483}, 'time_stamp': 'Tue May 21 18:36:06 2024', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': 'b6b033fb-40ff-41d9-9d84-7faf42af17c9', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'Testco MSA Tau.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 804, 'source_name': 'Testco MSA Tau.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 804, 'evidence_stop_char': 1206, 'source_name': 'Testco MSA Tau.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1607, 'source_name': 'Testco MSA Tau.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1607, 'evidence_stop_char': 2010, 'source_name': 'Testco MSA Tau.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2010, 'evidence_stop_char': 2412, 'source_name': 'Testco MSA Tau.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2814, 'source_name': 'Testco MSA Tau.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'Testco MSA Tau.pdf': [5, 4, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Tau.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n",
      "update: compare with evience-  {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n",
      "update: sources -  [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'Testco MSA Tau.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n"
     ]
    }
   ],
   "source": [
    "print(\"update: identified the following msa doc id: \", msa_doc_ids)\n",
    "\n",
    "for i, doc_id in enumerate(msa_doc_ids):\n",
    "\n",
    "  print(\"\\n\")\n",
    "  docs = msa_docs[i]\n",
    "  if os.sep in docs:\n",
    "    docs = docs.split(os.sep)[-1]\n",
    "\n",
    "  print (i+1, \"Reviewing MSA - \", doc_id, docs)\n",
    "\n",
    "  doc_filter = {\"doc_ID\": [doc_id]}\n",
    "  termination_provisions = q.text_query_with_document_filter(\"termination\", doc_filter)\n",
    "\n",
    "  sources = prompter.add_source_query_results(termination_provisions)\n",
    "\n",
    "  response = prompter.prompt_with_source(\"What is the notice for termination for convenience?\")\n",
    "\n",
    "  stats = prompter.evidence_comparison_stats(response)\n",
    "  ev_source = prompter.evidence_check_sources(response)\n",
    "\n",
    "  for i, resp in enumerate(response):\n",
    "    print(\"update: llm response - \", resp)\n",
    "    print(\"update: compare with evience- \", stats[i][\"comparison_stats\"])\n",
    "    print(\"update: sources - \", ev_source[i][\"source_review\"])\n",
    "\n",
    "  prompter.clear_source_materials()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eiM4sefL2SHv"
   },
   "outputs": [],
   "source": [
    "print(\"\\nupdate: Prompt state saved at: \", os.path.join(LLMWareConfig.get_prompt_path(), prompter.prompt_id))\n",
    "\n",
    "prompter.save_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "C3T-qhcM2gc0"
   },
   "outputs": [],
   "source": [
    "csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv()\n",
    "\n",
    "print(\"\\nupdate: CSV output for human review - \", csv_output)\n",
    "\n",
    "return 0"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0c6b1a2f4b6a4602bf5760d9e515f422": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0e66bf7f285343ea98f0456929049e72",
      "max": 3673967904,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3a5c51a6dd8e4f8a9fda6f11d370529a",
      "value": 3673967904
     }
    },
    "0e66bf7f285343ea98f0456929049e72": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1abee224a04142a798cc9f976fde5f41": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3a5c51a6dd8e4f8a9fda6f11d370529a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "3bb7c2beaf114d14a0b6e63947c25688": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_93a0447906784d72b7b2b4a5971c66d9",
      "placeholder": "​",
      "style": "IPY_MODEL_a94fb32c43794d6b87f15c22714c68f1",
      "value": " 3.67G/3.67G [01:09&lt;00:00, 67.8MB/s]"
     }
    },
    "58e60eec487640b3a552761db277c105": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "73f01dd1f62148ccbc03fd953927076b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1abee224a04142a798cc9f976fde5f41",
      "placeholder": "​",
      "style": "IPY_MODEL_e96669f232d04c079c91a1121e7dedb5",
      "value": "dragon-yi-6b-q4_k_m.gguf: 100%"
     }
    },
    "750db138aba04233b7552a809fe028e7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_73f01dd1f62148ccbc03fd953927076b",
       "IPY_MODEL_0c6b1a2f4b6a4602bf5760d9e515f422",
       "IPY_MODEL_3bb7c2beaf114d14a0b6e63947c25688"
      ],
      "layout": "IPY_MODEL_58e60eec487640b3a552761db277c105"
     }
    },
    "93a0447906784d72b7b2b4a5971c66d9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a94fb32c43794d6b87f15c22714c68f1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e96669f232d04c079c91a1121e7dedb5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "state": {}
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
